# -*- coding: utf-8 -*-
import scrapy

from TencentSpider.items import TencentspiderItem


class JobPositionSpider(scrapy.Spider):
    name = 'job_position'
    allowed_domains = ['tencent.com']
    start_urls = ['https://careers.tencent.com/search.html?query=ci_2&keyword=python']

    def parse(self, response):
        for recruit in response.xpath('//div[@class="recruit-list"]/a'):
            item = TencentspiderItem()
            item['title'] = recruit.xpath('./h4[@class="recruit-title"]/text()').extract_first()
            item['publish_date'] = recruit.xpath('./p[@class="recruit-tips"]/span/text()').extract()[-1]
            item['content'] = recruit.xpath('./p[@class="recruit-text"]/text()').extract_first()
            yield item
        # 计算下一页链接
        page_number_list = response.xpath(
            '//div[@class="page-number"]/ul/li[contains(@class, "page-li")]/span/text()').extract()
        max_page_number = max(int(x) for x in page_number_list if x.isdigit())
        this_page_number = int(response.xpath(
            '//div[@class="page-number"]/ul/li[@class="page-li active"]/span/text()').extract_first())
        next_page_number = this_page_number + 1
        if next_page_number <= max_page_number:
            next_link = 'https://careers.tencent.com/search.html?query=ci_2&keyword=python&index=' + str(
                next_page_number)
            print("next_link >>> " + next_link)
            yield scrapy.Request(next_link, callback=self.parse)
